package sky.demo.ml.word2vector;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;

import org.deeplearning4j.text.tokenization.tokenizer.TokenPreProcess;
import org.deeplearning4j.text.tokenization.tokenizer.Tokenizer;
import org.fnlp.nlp.cn.tag.CWSTagger;

/**
 * 
 * @author zhoujinhuang
 *
 */
public class FNLPTokenizer implements Tokenizer {
	private ArrayList<String> tag2List;
	Iterator<String> iterator;

	public FNLPTokenizer(CWSTagger tag, String tokens) {
		tag2List = tag.tag2List(tokens);
		if (tag2List == null) {
			iterator = Collections.emptyIterator();
		} else {
			iterator = tag2List.iterator();
		}
	}

	public FNLPTokenizer(CWSTagger tag, InputStream toTokenize) throws IOException {
		InputStreamReader inputStreamReader = new InputStreamReader(toTokenize, "UTF-8");
		BufferedReader bufferedReader = new BufferedReader(inputStreamReader);
		String readLine = bufferedReader.readLine();
		tag2List = new ArrayList<String>();
		while (readLine != null) {
			tag2List.addAll(tag.tag2List(readLine));
		}
		iterator = tag2List.iterator();
	}

	private TokenPreProcess tokenPreProcess;

	@Override
	public boolean hasMoreTokens() {
		return iterator.hasNext();
	}

	@Override
	public int countTokens() {
		return tag2List.size();
	}

	@Override
	public String nextToken() {
		String base = iterator.next();
		if (tokenPreProcess != null)
			base = tokenPreProcess.preProcess(base);
		return base;
	}

	@Override
	public List<String> getTokens() {
		List<String> tokens = new ArrayList<>();
		while (hasMoreTokens()) {
			tokens.add(nextToken());
		}
		return tokens;
	}

	@Override
	public void setTokenPreProcessor(TokenPreProcess tokenPreProcessor) {
		this.tokenPreProcess = tokenPreProcessor;

	}

}
